{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "# import argparse\n",
    "# import sys\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "#设置tf日志级别为info\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def swish(x):\n",
    "  return x * tf.nn.sigmoid(x)\n",
    "\n",
    "\n",
    "def selu(x):\n",
    "  with tf.name_scope('elu') as scope:\n",
    "    alpha = 1.6732632423543772848170429916717\n",
    "    scale = 1.0507009873554804934193349852946\n",
    "    return scale*tf.where(x>=0.0, x, alpha*tf.nn.elu(x))\n",
    "\n",
    "def relu(x):\n",
    "    return tf.nn.relu(x)\n",
    "\n",
    "def activation(x):\n",
    "#  return selu(x)\n",
    "#  return relu(x)\n",
    "#  return tf.nn.sigmoid(x)\n",
    "#  return tf.nn.elu(x)\n",
    "  return swish(x)\n",
    "\n",
    "def initialize(shape, stddev=0.1):\n",
    "  return tf.truncated_normal(shape, stddev=stddev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-3-782d7014abb4>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ../data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ../data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting ../data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ../data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '../data/'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_learning_rate = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建立神经网络，绘制计算图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#输入\n",
    "x = tf.placeholder(tf.float32, [None, 784])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create the model\n",
    "# 第一层隐层，神经元数为100\n",
    "L1_units_count = 100\n",
    "\n",
    "#tf.shape(x)  [100, 784]\n",
    "# 指数级学习率衰减\n",
    "epoch_steps = tf.cast(tf.math.divide(60000, tf.shape(x)[0]), tf.int64)\n",
    "global_step = tf.train.get_or_create_global_step()\n",
    "current_epoch = global_step//epoch_steps\n",
    "decay_times = current_epoch \n",
    "current_learning_rate = tf.multiply(init_learning_rate, \n",
    "                                    tf.pow(0.575, tf.to_float(decay_times)))\n",
    "\n",
    "W_1 = tf.Variable(initialize([784, L1_units_count], \n",
    "                             stddev=np.sqrt(2/784)))\n",
    "b_1 = tf.Variable(tf.constant(0.001, shape=[L1_units_count]))\n",
    "logits_1 = tf.matmul(x, W_1) + b_1\n",
    "output_1 = activation(logits_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 第二层隐层，神经元数为10\n",
    "L2_units_count = 10 \n",
    "W_2 = tf.Variable(initialize([L1_units_count, \n",
    "                              L2_units_count], \n",
    "                             stddev=np.sqrt(2/L1_units_count)))\n",
    "b_2 = tf.Variable(tf.constant(0.001, shape=[L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "\n",
    "y = logits_2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义损失和优化器\n",
    "\n",
    "损失为tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)的均值\n",
    "\n",
    "此外，损失函数还加了L2正则化，通过加入一个正则项来防止过拟合（data overfitting）tf.nn.l2_loss(）\n",
    "\n",
    "优化器选取tf.train.AdamOptimizer(learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-12-9a4a4f60a81c>:12: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "step 100, entropy loss: 0.471254, l2_loss: 295.858002, total loss: 0.483088\n",
      "0.9275\n",
      "step 200, entropy loss: 0.117890, l2_loss: 409.046661, total loss: 0.134252\n",
      "0.9455\n",
      "step 300, entropy loss: 0.264348, l2_loss: 493.120117, total loss: 0.284073\n",
      "0.9578\n",
      "step 400, entropy loss: 0.090640, l2_loss: 575.954224, total loss: 0.113678\n",
      "0.9589\n",
      "step 500, entropy loss: 0.217881, l2_loss: 692.878845, total loss: 0.245596\n",
      "0.9588\n",
      "step 600, entropy loss: 0.025031, l2_loss: 754.556091, total loss: 0.055213\n",
      "0.9646\n",
      "step 700, entropy loss: 0.055365, l2_loss: 742.316956, total loss: 0.085058\n",
      "0.9688\n",
      "step 800, entropy loss: 0.090895, l2_loss: 738.058289, total loss: 0.120417\n",
      "0.9716\n",
      "step 900, entropy loss: 0.032683, l2_loss: 742.392578, total loss: 0.062379\n",
      "0.9679\n",
      "step 1000, entropy loss: 0.059372, l2_loss: 746.720520, total loss: 0.089241\n",
      "0.9711\n",
      "step 1100, entropy loss: 0.023218, l2_loss: 750.615234, total loss: 0.053243\n",
      "0.9673\n",
      "step 1200, entropy loss: 0.084649, l2_loss: 744.206055, total loss: 0.114417\n",
      "0.9717\n",
      "step 1300, entropy loss: 0.082175, l2_loss: 735.992859, total loss: 0.111614\n",
      "0.9774\n",
      "step 1400, entropy loss: 0.033467, l2_loss: 727.525269, total loss: 0.062568\n",
      "0.9769\n",
      "step 1500, entropy loss: 0.009977, l2_loss: 718.817078, total loss: 0.038729\n",
      "0.9769\n",
      "step 1600, entropy loss: 0.033480, l2_loss: 715.828125, total loss: 0.062113\n",
      "0.9783\n",
      "step 1700, entropy loss: 0.030711, l2_loss: 705.471008, total loss: 0.058930\n",
      "0.9783\n",
      "step 1800, entropy loss: 0.035048, l2_loss: 694.109131, total loss: 0.062813\n",
      "0.9772\n",
      "step 1900, entropy loss: 0.041076, l2_loss: 687.986755, total loss: 0.068596\n",
      "0.9787\n",
      "step 2000, entropy loss: 0.013775, l2_loss: 682.019165, total loss: 0.041056\n",
      "0.9793\n",
      "step 2100, entropy loss: 0.066283, l2_loss: 676.179260, total loss: 0.093330\n",
      "0.9802\n",
      "step 2200, entropy loss: 0.028461, l2_loss: 672.293823, total loss: 0.055353\n",
      "0.9788\n",
      "step 2300, entropy loss: 0.024821, l2_loss: 663.620178, total loss: 0.051366\n",
      "0.98\n",
      "step 2400, entropy loss: 0.008282, l2_loss: 657.159485, total loss: 0.034569\n",
      "0.9794\n",
      "step 2500, entropy loss: 0.013622, l2_loss: 652.107056, total loss: 0.039707\n",
      "0.9791\n",
      "step 2600, entropy loss: 0.025942, l2_loss: 647.538879, total loss: 0.051843\n",
      "0.9799\n",
      "step 2700, entropy loss: 0.004728, l2_loss: 642.901978, total loss: 0.030444\n",
      "0.9811\n",
      "step 2800, entropy loss: 0.008886, l2_loss: 639.275452, total loss: 0.034457\n",
      "0.9802\n",
      "step 2900, entropy loss: 0.005383, l2_loss: 633.498047, total loss: 0.030723\n",
      "0.9797\n",
      "step 3000, entropy loss: 0.014200, l2_loss: 628.618958, total loss: 0.039345\n",
      "0.9801\n"
     ]
    }
   ],
   "source": [
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.nn.l2_loss(W_1) + tf.nn.l2_loss(W_2)\n",
    "\n",
    "total_loss = cross_entropy + 4e-5*l2_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(current_learning_rate)\n",
    "\n",
    "gradients = optimizer.compute_gradients(total_loss)\n",
    "\n",
    "train_step = optimizer.apply_gradients(gradients)\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(\n",
    "    current_learning_rate).minimize(\n",
    "    total_loss, global_step=global_step)\n",
    "# weight decay      \n",
    "#0 0 0 0 1 0 0 0 0\n",
    "#0 1 0 0 0 0 0 0 0\n",
    "# 还记得不？不要把graph定义写到图里面\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()\n",
    "# Train\n",
    "for step in range(3000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.01\n",
    "  _, loss, l2_loss_value, total_loss_value, current_lr_value = \\\n",
    "      sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss, \n",
    "                current_learning_rate], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, \n",
    "                          init_learning_rate:lr})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    #print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 如何修改隐层数量，修改后会起到什么样的效果"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "理论上来讲，增加隐层数可以降低网络误差，提高精度（其实效果不是很明显），但也使网络复杂化，从而增加了网络的训练时间和出现“过拟合”的倾向。  \n",
    "\n",
    "一般来讲应设计神经网络应优先考虑3层网络（即有1个隐层）。  \n",
    "\n",
    "如果靠增加隐层节点数来获得较低的误差，其训练效果要比增加隐层数更容易实现。  \n",
    "\n",
    "对于没有隐层的神经网络模型，实际上就是一个线性或非线性（取决于输出层采用线性或非线性转换函数型式）回归模型。  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 如何神经元个数，起到了什么样的效果"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "例如，如果有一个在二维平面上的二分类问题。我们可以训练3个不同的神经网络，每个网络都只有一个隐层，但是每层的神经元数目不同："
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![mlp](../pics/mlp.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "更大的神经网络可以表达更复杂的函数。数据是用不同颜色的圆点表示他们的不同类别，决策边界是由训练过的神经网络做出的。  \n",
    "\n",
    "在上图中，可以看见有更多神经元的神经网络可以表达更复杂的函数。然而这既是优势也是不足，优势是可以分类更复杂的数据，不足是可能造成对训练数据的过拟合。  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 如何在模型中添加L1/L2正则化，正则化起什么作⽤"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "正则化在模型中起到了弥补过拟合的问题。\n",
    "\n",
    "#### 什么是过拟合？\n",
    "\n",
    "过拟合（Overfitting）是网络对数据中的噪声有很强的拟合能力，而没有重视数据间原有的基本关系。  \n",
    "\n",
    "换句话来讲，由于模型精度过高，在你和的过程中，将噪声样本也计算在内，实质上，噪声在拟合的时候应该被忽略。\n",
    "\n",
    "举例来说，有20个神经元隐层的网络拟合了所有的训练数据，但是其代价是把决策边界变成了许多不相连的红绿区域。  \n",
    "\n",
    "而有3个神经元的模型的表达能力只能用比较宽泛的方式去分类数据。它将数据看做是两个大块，并把个别在绿色区域内的红色点看做噪声。在实际中，这样可以在测试数据中获得更好的泛化（generalization）能力。\n",
    "\n",
    "无论以上哪一种正则化方式，正则化基本的思想都是希望通过限制权重的大小，使得模型不能任意拟合训练数据中的随机噪声。  \n",
    "\n",
    "但是这两种正则化的方法有很大区别：  \n",
    "首先，L1正则化会让参数变得稀疏，而L2正则化不会。所谓参数变得更加稀疏是指会有更多的参数变成0，这样可以达到类似特征选取的功能。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 权重使⽤不同的初始化⽅式对模型有什么影响"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. 最原始的初始化：全部为固定值  \n",
    "2. 稍好些的初始化：服从固定方差的独立高斯分布\n",
    "3. Xavier初始化或MSRA初始化\n",
    "\n",
    "#### 1. 全部为固定值  \n",
    "权重全部取固定值，会导致模型的最终训练效果不好。如果固定值为0，那么每一层的输出是一样的，而且在bp的时候，每一层内的神经元也是相同的，因为他们的gradient相同。最终导致模型不收敛。  \n",
    "\n",
    "如果固定值非零，可以训练模型，但是速度很慢，为了解决这个问题，服从有固定方差的独立高斯分布的方法被采用。\n",
    "\n",
    "#### 2. 服从固定方差的独立高斯分布\n",
    "W-(N, α^2)\n",
    "这个方法提升了训练速度；\n",
    "然而， 随着层级的加深，高斯分布初始化的参数容易出现过大或过小的问题，最终的解决方案是Xavier初始化或MSRA初始化  \n",
    "\n",
    "#### 3. Xavier初始化或MSRA初始化  \n",
    "Xavier初始化服从参数为n的均匀分布或独立高斯分布, 搭配tanh激活函数效果更佳  \n",
    "![Xavier](../pics/Xavier.png)\n",
    "MSRA初始化服从参数为n的独立高斯分布， 搭配Relu激活函数效果更佳\n",
    "![MSRA](../pics/MSRA.png)\n",
    "\n",
    "![pic](../pics/pic.png)\n",
    "\n",
    "![graph](../pics/graph.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
